#!/usr/bin/perl
use strict;
use warnings;
use 5.010;

use LWP::Simple;
use WWW::Mechanize;
use HTTP::Response;
use HTML::TreeBuilder;
use IO::File;
use IO::Handle;

my %subsubcat;
my $base_url = "http://gtrnadb.ucsc.edu/";

my $bot = WWW::Mechanize->new(stack_depth => 0, cookie_jar => undef);

say "Getting first page ...";
$bot->get($base_url);

my $tree = HTML::TreeBuilder->new_from_content($bot->content);
my @headers = $tree->look_down("_tag" => "td", "class" => qr/table_header1/);

my $cat, my $subcat;
foreach my $block (@headers) {
    $cat = (($block->content_list)[1]);

    my @b = $block->parent->parent->look_down("_tag" => "td", sub { $_[0]->look_down("_tag" => "b", sub { ($_[0]->content_list)[0] =~ m/^[ A-Za-z-]+$/ }) or $_[0]->look_down("_tag" => "a", "href" => qr!^[A-Z][a-zA-Z0-9_-]+/$!) } );
    foreach my $block2 (@b) {
        if(my @bb = $block2->look_down("_tag" => "b", sub { ($_[0]->content_list)[0] =~ m/^[A-Za-z-]+$/ })) {
            $subcat = (($bb[0])->content_list)[0];
        }
        else {
            my $name = ((($block2->content_list)[0]->content_list)[0]);
            if(defined($name)) {
                $name =~ s/^\s+//;
                $name =~ s/\s+$//;

                $subsubcat{$name} = { "cat" => $cat, "subcat" => $subcat };
            }
        }
    }
}
$tree->delete;

my $trnas = 0, my $bad = 0, my $num = 0;
my $first = "";

my $csv_file = IO::File->new("./database.csv", "+>") or die "Unable to create database.csv: $!";

my @links = $bot->find_all_links(url_regex => qr!^[A-Z][a-zA-Z0-9_-]+/$!);
foreach my $l (@links) {
    print " "x(length($first)),"\r";

    $first = "Getting ".$l->text." [".(++$num)."/".scalar(@links)."]";
    print $first;
    flush STDOUT;

    my @bits = (get($base_url.$l->url.substr($l->url, 0, -1)."-structs.html") =~ m{<PRE.*?>(.*?)</PRE>}gs);
    foreach my $pre (@bits) {
            my @list = $pre =~ m!^([^ ]+) .*Length: ([0-9]+) bp.+Type: ([^ \t]+)[ \t]+Anticodon: ([?A-Za-z]+) at ([0-9]+).*Seq: ([A-Za-z]+).Str: ([.><]+).*$!gs;

            if(!@list || $list[3] eq "???") {
                $bad++;
            }
            else {
                $list[0] =~ s/,//g;

                $csv_file->print(${$subsubcat{$l->text}}{"cat"}.",".${$subsubcat{$l->text}}{"subcat"}.",".$l->text.",".substr($list[0], 1).",".$list[1].",".$list[2].",".$list[3].",".$list[4].",".$list[5].",".$list[6]."\n") or die "Unable to write in database.csv: $!";
                $trnas++;
            }
    }
    print " "x(100-length($first)),"$trnas tRNAs added, $bad rejected.\r";
    flush STDOUT;
}
print "\n";

$csv_file->close or die "Unable to close database.csv: $!";
